<?php 
	/**
	 * 赶集网抓取的通用方法
	 */

	//注册赶集网id，已经不好使了，不过这个也没用到
	function registerGanjiId()
	{
		$url = "http://bj.ganji.com/user/register.php";
		$parameter = array("username" => "testa".time() ,
							"expireDays" => 0,
							"email" => "",  
							"source" => "passport",    
							"password" => "1qa2ws" ,
							"password2" => "1qa2ws" ,
							"next" => "http://bj.ganji.com/" ,
							"affirm" => "on" ,
					);
		$opt_arr = array(CURLOPT_REFERER => "http://bj.ganji.com",  //referer
						 CURLOPT_HEADER => true,						//输出中包含头信息				
					);
		printlog("using:".$parameter['username']." as register name", "register");			
		$page = makeRequest($url, 
							"POST", 
							$parameter, 
							true, 
							ROOT_PATH."/ganji/cookie/ganji_register_cookie.txt",
							"", 
							$opt_arr
							);
		if (strpos($page, "register_success") !== false) {
			preg_match('/user_id%22%3A%22([0-9]+)%22%2C/', $page, $arr);
			$id = $arr[1];
			printlog("register success, newid is $id", "register");
			return $id;
		}
		else{
			return false;
		}
	}
		 
	//自动保存新注册的id，同样没用到
	function saveGanjiId($new_ganji_id)
	{
		global $conn;
		$today = date("Y-m-d", time());
		$now = date("Y-m-d H:i:s", time());
		$info = $now."\t".$new_ganji_id."\n";
		//保存至文件（防止数据库挂掉，备查）
		file_put_contents(ROOT_PATH."/ganji/log/register_log/$today.log", $info, FILE_APPEND);
		//保存至数据库
		$arr = array("site" => "ganji",
					"ymd" => $today,
					"register_time" => $now,
					"source_user_id" => $new_ganji_id
				);
		$sql = Sql::insert($arr, "register_log");
		$flag = $conn->query($sql);
		return $flag;
	}
	
	/**
	 * 最主要的方法，循环所有搜索条件（城市、性别、年龄），抓取搜索结果，分析用户属性，最后入库
	 */
	function ganji_crawl()
	{
		global $conn, $yesterday;

		//获取搜索条件之————城市详细信息
		$city_arr = array("bj"=>array("chname"=>"北京", "province"=>"北京", "use_age"=>1),
							"sh"=>array("chname"=>"上海", "province"=>"上海", "use_age"=>1),
							"tj"=>array("chname"=>"天津", "province"=>"天津", "use_age"=>1),
							"cq"=>array("chname"=>"重庆", "province"=>"重庆", "use_age"=>1),
						);
		$sql = "select * from ganji_city ";
		$query = $conn->query($sql);
		while ($row = $conn->fetch_array()) {
			$city_chname = $row['city_chname'];
			$city_enname = $row['city_enname'];
			$province_name = $row['province_name'];
			$use_age = $row['use_age'];
			$city_arr[$city_enname] = array("chname" => $city_chname, 
											"province" => $province_name,
											"use_age" => $use_age,
										);
		}
		//定义搜索条件之————性别
		$sex_arr = array("gg","mm");
		//定义搜索条件之————年龄
		$ages_arr = array(18,19,20,21,22,23,24,25,26,27,28,29,30,31,41,61);
		
		//对所有城市循环
		foreach ($city_arr as $city_enname => $city_value){
			//对性别循环（男和女）
			foreach ($sex_arr as $sex){
				$use_age = $city_value['use_age'];
				//对于用户较多的城市，再按照年龄循环
				if ($use_age) {
					$start_age = $end_age = 0;
					foreach ($ages_arr as $end_age){
						if (!empty($start_age)) {
							dowithCondition($city_enname, $start_age, $end_age-1, $sex);
							//printlog($city."_".$start_age."_".($end_age-1)."_".$sex);
							sleep(3);
						}
						$start_age = $end_age;
					}
				}
				else{
					dowithCondition($city_enname, GANJI_MIN_AGE, GANJI_MAX_AGE, $sex);
				}
			}
		}
	}
	
	/**
	 * 抓取一个搜索结果页并保存
	 * @param $city 城市 bj
	 * @param $age 年龄 18
	 * @param $sex 性别 gg or mm
	 * @param $page 第几页
	 */
	function crawlSinglePage($city, $start_age, $end_age, $sex, $page)
	{
		global $page_dir;
		//while条件的说明：
		//1，页面内容不足，可能抓取的不是正常的搜索结果页
		//2，抓取到的页面中未发现html标签，说明没有抓取完全
		//如果出现这两个条件，则重新抓
		do{
			$url = "http://love.ganji.com/$city/$sex/?&age_start=$start_age&age_end=$end_age&photo=0&order=3&list=2&page=$page";
			$opt_arr = array(CURLOPT_REFERER => "http://love.ganji.com/bj/mm/",  //referer			
							);
			printlog("start crawl url : $url");
			$html = makeRequest($url, 
								"GET", 
								"", 
								false, 
								"", 
								"", 
								$opt_arr
								);
			sleep(3);
		} while(strlen($html) < 10000 || !checkCompleteDownload($html));
		
		file_put_contents($page_dir."/".$city."_".$start_age."_".$end_age."_".$sex."_".$page.".html", $html);
	}
	
	/**
	 * 从抓取的搜索页中分析数据，返回一个分析结果的数组
	 * 结果数组目前有两种形式：code=1表明正常，value即为提取的用户数据；code=-1表明无搜索结果
	 * 返回数组格式：
	 * @param $city
	 * @param $start_age
	 * @param $end_age
	 * @param $sex
	 * @param $page
	 */
	function parseSinglePage($city, $start_age, $end_age, $sex, $page)
	{
		global $page_dir;
		$retarr = array();
		$file = $page_dir."/".$city."_".$start_age."_".$end_age."_".$sex."_".$page.".html";
		if (!file_exists($file)) {
			crawlSinglePage($city, $start_age, $end_age, $sex, $page);
		}
		$html = file_get_contents($file);
		//特殊情况：搜索结果为空，或者循环至最后一页的下一页
		if (strpos($html, "没有找到符合您搜索条件的会员")) {
			$retarr['code'] = "-1";
			$retarr['value'] = "搜索结果为空，或循环至最后一页的下一页";
			return $retarr;
		}
		
		//从抓取到的html中匹配用户
		$html = str_replace(array("\n","\r", " ", "\t"), "", $html);
		//preg_match_all('/<img src="http:.+width="118" height="150".+\s+<\/a>\s+<div class="infor-user">.+\s+.+/', $html, $arr, PREG_SET_ORDER);

		preg_match_all('/<divclass="show-tx-new".+?岁/', $html, $arr, PREG_SET_ORDER);
		//var_dump($arr);exit;
		/* 原始html--有照片的
		 * <img src="http://image.ganjistatic1.com/gjfs01/M01/1F/C0/wKhxwE54SrXDGEAzAAEMqluw,BY014_120-150_8-5.jpg" width="118" height="150">
                        		                        		</a>
                            <div class="infor-user">
                            	<p><a href="/65044076.htm" target="_blank" title="耶梅花" >耶梅花</a></p><p>18岁 . 大专 . 北京
		 */
		/** 原始html--没照片的
		 * 	<a href="/65433538.htm" target="_blank">
                        																																	<img src="http://sta.ganjistatic1.com/src/image/personals/user/women-2.jpeg" width="118" height="150"/>
																													                        		</a>
                            <div class="infor-user">
                            	<p><a href="/65433538.htm" target="_blank" title="昱诺" >昱诺</a></p><p>18岁 . 高中 . 北京 
		 */
		//printlog($arr); //最多有30个
		//exit;
		$headcount = count($arr);
		
		//如果抓下来的页面没有数据，重新抓一次，其实crawl的时候有过容错处理，这儿应该执行不到
		if ($headcount == 0) {
			printlog("页面  $file 未发现用户，重新抓，其实crawl的时候有过容错处理，这儿应该执行不到才对", "warning");
			recrawlSinglePage($city, $start_age, $end_age, $sex, $page);
			parseSinglePage($city, $start_age, $end_age, $sex, $page);
			return;
		}
		
		$retarr['code'] = 1;
		for ($i = 0; $i < $headcount; $i++) {
			//preg_match('/[0-9]+/', $arr[$i][0], $id_arr);
			//$topIds[$city][$start_age][$sex][$i]['id'] = $id_arr[0];
			//id and name
			//<a href="/56432530.htm" target="_blank" title="不喝孟婆汤"
			preg_match_all('/\/([0-9]+)\.htm"(.*?)title="(.*?)"/', $arr[$i][0], $id_arr);
			//var_dump($id_arr);exit;
			$id = $id_arr[1][0];
			$retarr['value'][$i]['id'] = $id;
			$name = $id_arr[3][0];
			$retarr['value'][$i]['name'] = $name;
			//age
			preg_match('/[0-9]+岁/', $arr[$i][0], $age_arr);
			$retarr['value'][$i]['age'] = substr($age_arr[0], 0, strlen($age_arr[0])-3);
			//has_photo
			if (strpos($arr[$i][0], "men") > 0 || strpos($arr[$i][0], "women") > 0) {
				$retarr['value'][$i]['has_photo'] = 0;
			}
			else{
				$retarr['value'][$i]['has_photo'] = 1;
			}
		}
		//var_dump($retarr);exit;
		return $retarr;
	}
	
	/**
	 * 外层方法，处理某种搜索条件
	 */
	function dowithCondition($city, $start_age, $end_age, $sex)
	{
		global $conn, $yesterday;
		//获取城市所在的省份
		$city_arr = array("bj"=>"北京",
						"sh"=>"上海",
						"tj"=>"天津",
						"cq"=>"重庆",
						);
		$sql = "select * from ganji_city ";
		$query = $conn->query($sql);
		while ($row = $conn->fetch_array()) {
			$city_enname = $row['city_enname'];
			$province_name = $row['province_name'];
			$city_arr[$city_enname] = $province_name;
		}
		
		//从第1页开始处理
		$page = 1;
		//while($true){ //每次翻一页，直到到达最后一页
		while($page<=50){ //改成最多抓取50页，防止马甲注册
			printlog("start to do with a new page ----  $city, $start_age, $end_age, $sex, $page ");
			//格式化该搜索页
			$html_parse_arr = parseSinglePage($city, $start_age, $end_age, $sex, $page);
			
			//没有找到符合条件的用户
			if ($html_parse_arr['code'] == -1) {
				return -1;
			}
			//找到符合条件的用户
			else if($html_parse_arr['code'] == 1){
				$html_userdata_arr = $html_parse_arr['value'];
				$exist_num = 0;
				//循环该结果页中发现的所有用户
				for ($i = 0; $i < count($html_userdata_arr); $i++) {
					$head = $html_userdata_arr[$i];
					$id = $head['id'];
					$age = $head['age'];
					$name = $head['name'];
					$has_photo = $head['has_photo'];
					$sql = "select * from ganji_user  where id=$id ";
					$exist = $conn->has_record($sql);
					if ($exist) {
						//表里有了
						printlog("user exists: id $id , $city, $start_age, $end_age, $sex, $page");
						$exist_num ++;
						//连续2个都存在，才判断为到达昨天抓取的地方，停止循环。否则可能会因为赶集的bug而提前终止
						if ($exist_num >= 2) {
							break;
						}
						else{
							continue;
						}
					}
					else 
					{
						//表里没有，保存搜索结果
						$exist_num = 0;
						$str = $city."\t".$age."\t".$sex."\t".$has_photo."\t".$name."\t".$id."\n";
						//插入ganji_user 表
						$arr_insert = array("id" => $id,
										"city" => $city,
										"province" => $city_arr[$city],
										"age" => $age,
										"sex" => $sex,
										"has_photo" => $has_photo,
										"name" => $name,
										"ymd" => $yesterday,
										"updatetime" => date("Y-m-d H:i:s"),
										);
						$sql = Sql::insert($arr_insert, "ganji_user" );
						$conn->query($sql);

						//插入competitor 表
						$gender = ($sex=="mm")?0:1; //女为0， 男为1
						$tblName = getTblName($yesterday);
						$arr_insert = array("source_user_id" => $id,
											"city" => $city,
											"province" => $city_arr[$city],
											"age" => $age,
											"gender" => $gender,
											"has_photo" => $has_photo,
											"fetch_date" => $yesterday,
											"`from`" => "ganji",
											);
						$sql = Sql::insert($arr_insert, $tblName );
						if($conn->query($sql)){
							printlog("new user: id $id ");
						}
					}	
				}
				
				//检查该页显示的人数是否达到每页上限，并且没有是否发现相同的id，如果满足的话继续分析下一页
				if (count($html_userdata_arr) == GANJI_HEADCOUNT_PER_PAGE && $exist_num < 2){
					$page++;
					continue;
				}
				else if (count($html_userdata_arr) > GANJI_HEADCOUNT_PER_PAGE){
					printlog("当前页采集到的用户个数超过最大值，是不是bt的赶集改页面了？" , "warning");
				}
				else{
					break;
				}
			}
		}
	}
	/**
	 * 删除某个已经抓取下来的页面，并重新抓取
	 * @param $city
	 * @param $start_age
	 * @param $end_age
	 * @param $sex
	 * @param $page
	 */
	function recrawlSinglePage($city, $start_age, $end_age, $sex, $page)
	{
		printlog("删除已经抓下来的页面  $city, $start_age, $end_age, $sex, $page", "warning");
		global $page_dir;
		$file = $page_dir."/".$city."_".$start_age."_".$end_age."_".$sex."_".$page.".html";
		if (file_exists($file)) {
			copy ($file, $page_dir."/".$city."_".$start_age."_".$end_age."_".$sex."_".$page.".bak.html");
			unlink ($file);
			crawlSinglePage($city, $start_age, $end_age, $sex, $page);
		}
	}


?>